from tools.redis_queue import RedisQueue
from configs.cfg import *
import time
import pymysql


# 新建队列名为rq

class GetTasks:
    q = RedisQueue('rq')

    def get_tasks(self):
        while True:
            if self.q.qsize() > 0:
                url = self.q.get_wait()
                yield url

    def put_tasks(self):
        while True:
            # result = q.get_nowait()
            print(f'当前队列的长度为{self.q.qsize()}')
            if self.q.qsize() <= 10:
                time.sleep(2)
                conn = pymysql.connect(host=mysql_host, user=mysql_name, password=mysql_pwd,
                                       database='spiderdb', port=mysql_port, charset='utf8')
                # 获取执行工具
                cur = conn.cursor()
                sql = 'SELECT website FROM websiteaddresses WHERE is_crawl IS Null LIMIT 1000'
                cur.execute(sql)
                datas = cur.fetchall()
                print(datas)
                for i in datas:
                    print(i)
                    self.q.put(i[0])


if __name__ == '__main__':
    handle = GetTasks()
    handle.get_tasks()
    # handle.put_tasks()
